{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 针对两条元路径，分别进行矩阵分解，然后生成UI.pkl和UIUI.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T10:45:46.708434Z",
     "start_time": "2020-09-19T10:45:46.335402Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "data_path=\"hin/data/u.data\"\n",
    "data_fields = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "# all data file\n",
    "data_df = pd.read_table(data_path, names=data_fields)\n",
    "\n",
    "# get user number\n",
    "n_users = max(data_df['user_id'].values)\n",
    "# get item number\n",
    "n_items = max(data_df['item_id'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T10:47:19.306232Z",
     "start_time": "2020-09-19T10:47:18.997892Z"
    }
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import coo_matrix\n",
    "# S = dok_matrix((5, 5), dtype=np.float32)\n",
    "data = np.ones((data_df.shape[0]))\n",
    "data=data_df.rating.values\n",
    "row = data_df.user_id-1\n",
    "col = data_df.item_id-1\n",
    "UI = coo_matrix((data, (row, col)), shape=(n_users, n_items))\n",
    "UIUI = UI.dot(UI.transpose()).dot(UI)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T10:47:21.476606Z",
     "start_time": "2020-09-19T10:47:21.344960Z"
    }
   },
   "outputs": [],
   "source": [
    "UIUI=UIUI.tocoo()\n",
    "UI=UI.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T10:47:59.757755Z",
     "start_time": "2020-09-19T10:47:57.615478Z"
    }
   },
   "outputs": [],
   "source": [
    "def save_M(M, M_str):\n",
    "    df=pd.DataFrame()\n",
    "    df['row']=M.row\n",
    "    df['col']=M.col\n",
    "    df['data']=M.data.astype(np.int)\n",
    "    df.to_csv(M_str,header=None,index=False,sep='\\t')\n",
    "    \n",
    "save_M(UIUI,'hin/UIUI.txt')\n",
    "save_M(UI,'hin/UI.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-01-22T05:17:45.486998Z",
     "start_time": "2020-01-22T05:17:44.807645Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T11:38:25.533596Z",
     "start_time": "2020-09-19T11:38:24.710797Z"
    }
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import math\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as data\n",
    "from tqdm import tqdm\n",
    "from utils import Interactions\n",
    "import os\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from IPython import embed\n",
    "\n",
    "IMPLICT=False\n",
    "SMALL=False\n",
    "\n",
    "# for reproducibility\n",
    "def seed_everything(seed=1234):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything()\n",
    "\n",
    "# To compute probalities\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def drop_df(df):\n",
    "    pos_cnt = df.groupby('user_id', as_index=False)['rating'].agg({\"pos_cnt\": 'sum'})\n",
    "    tot_cnt = df.groupby('user_id', as_index=False)['rating'].agg({\"tot_cnt\": 'count'})\n",
    "    df = pd.merge(df, pos_cnt, on=['user_id'], how='left')\n",
    "    df = pd.merge(df, tot_cnt, on=['user_id'], how='left')\n",
    "    df = df[(df.pos_cnt > 0) & (df.tot_cnt > df.pos_cnt)]\n",
    "    df = df.drop(['pos_cnt', 'tot_cnt'], axis=1)\n",
    "    return df\n",
    "\n",
    "def getDataLoader(data_path, batch_size=2048):\n",
    "    # load train data\n",
    "    data_fields = ['user_id', 'item_id', 'rating', 'timestamp']\n",
    "    # all data file\n",
    "    data_df = pd.read_table(data_path, names=data_fields)\n",
    "    if SMALL:\n",
    "        data_df = data_df.sample(n=int(len(data_df) * 0.1), replace=False)\n",
    "    if IMPLICT:\n",
    "        data_df.rating = (data_df.rating >= 4).astype(np.float32)\n",
    "    # ua_base = allData.sample(n=90570, replace=False)\n",
    "    df_train = data_df.sample(n=int(len(data_df) * 0.8), replace=False)\n",
    "    df_test = data_df.drop(df_train.index, axis=0)\n",
    "    if IMPLICT:\n",
    "        df_train=drop_df(df_train)\n",
    "        df_test = drop_df(df_test)\n",
    "    # get user number\n",
    "    n_users = max(set(data_df['user_id'].values))+1\n",
    "    # get item number\n",
    "    n_items = max(set(data_df['item_id'].values))+1\n",
    "\n",
    "    print(\"Initialize end.The user number is:%d,item number is:%d\" % (n_users, n_items))\n",
    "    train_loader = data.DataLoader(\n",
    "        Interactions(df_train,index_from_one=False), batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    test_loader = data.DataLoader(\n",
    "        Interactions(df_test,index_from_one=False), batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    loaders = {'train': train_loader,\n",
    "               'valid': test_loader}\n",
    "\n",
    "    return (n_users,n_items ), loaders\n",
    "\n",
    "class LFM(torch.nn.Module):\n",
    "    def __init__(self, n_users, n_items, n_factors=20, lr=0.1, weight_decay=0.001, sparse=False,topn=10, device=torch.device(\"cpu\")):\n",
    "        super(LFM, self).__init__()\n",
    "\n",
    "        self.n_users = n_users\n",
    "        self.n_items = n_items\n",
    "        self.device = device\n",
    "        self.topn=topn\n",
    "\n",
    "        # get factor number\n",
    "        self.n_factors = n_factors\n",
    "#         self.user_biases = nn.Embedding(self.n_users, 1, sparse=sparse)\n",
    "#         self.item_biases = nn.Embedding(self.n_items, 1, sparse=sparse)\n",
    "        self.user_embeddings = nn.Embedding(self.n_users, self.n_factors, sparse=sparse)\n",
    "        self.item_embeddings = nn.Embedding(self.n_items, self.n_factors, sparse=sparse)\n",
    "\n",
    "        self.sparse = sparse\n",
    "\n",
    "        self.optimizer = torch.optim.Adam(self.parameters(),\n",
    "                                   lr=lr, weight_decay=0.5)\n",
    "        self=self.to(self.device)\n",
    "\n",
    "\n",
    "    def forward(self, users, items):\n",
    "        users=users.to(self.device)\n",
    "        items = items.to(self.device)\n",
    "        try:\n",
    "            ues = self.user_embeddings(users)\n",
    "            uis = self.item_embeddings(items)\n",
    "\n",
    "#             preds = self.user_biases(users) # b 1\n",
    "#             preds += self.item_biases(items)# b 1\n",
    "            # preds += (self.dropout(ues) * self.dropout(uis)).sum(dim=1, keepdim=True)\n",
    "            preds= ((ues) * (uis)).sum(dim=1, keepdim=True)\n",
    "        except Exception as ex:\n",
    "            print(ex)\n",
    "            embed()\n",
    "            \n",
    "        return preds.squeeze()\n",
    "\n",
    "    def fit(self, loaders, epochs=5):\n",
    "        # training cycle\n",
    "        best_score = 0.\n",
    "        for epoch in range(epochs):\n",
    "            losses = {'train': 0., 'valid': 0}\n",
    "\n",
    "            for phase in ['train', 'valid']:\n",
    "\n",
    "                if phase == 'train':\n",
    "                    self.train()\n",
    "                else:\n",
    "                    self.eval()\n",
    "                pbar = tqdm(enumerate(loaders[phase]),\n",
    "                            total=len(loaders[phase]),\n",
    "                            desc='({0}:{1:^3})'.format(phase, epoch+1))\n",
    "                for batch_idx, ((row, col), val) in pbar:\n",
    "                # for batch_x, batch_y in loaders[phase]:\n",
    "                    self.optimizer.zero_grad()\n",
    "\n",
    "                    row = row.long()\n",
    "                    col = col.long()\n",
    "                    val = val.float().to(self.device)\n",
    "                    preds = self.forward(row, col)\n",
    "                    loss = nn.MSELoss(reduction='sum')(preds, val)\n",
    "\n",
    "                    losses[phase] += loss.item()\n",
    "                    batch_loss = loss.item() / row.size()[0]\n",
    "                    pbar.set_postfix(train_loss=batch_loss)\n",
    "\n",
    "                    with torch.set_grad_enabled(phase == 'train'):\n",
    "                        if phase == 'train':\n",
    "                            loss.backward()\n",
    "                            #                             scheduler.step()\n",
    "                            self.optimizer.step()\n",
    "\n",
    "                losses[phase] /= len(loaders[phase].dataset)\n",
    "            # print('epoch done')\n",
    "            # after each epoch check if we improved roc auc and if yes - save model\n",
    "            with torch.no_grad():\n",
    "                model.eval()\n",
    "\n",
    "                y_pred,y_true = [],[]\n",
    "\n",
    "                for ((row, col), val) in loaders['valid']:\n",
    "                    row = row.long()\n",
    "                    col = col.long()\n",
    "                    val = val.float()\n",
    "                    preds = self.forward(row, col)\n",
    "                    if IMPLICT:\n",
    "                        preds = sigmoid(preds.cpu().numpy())\n",
    "                    y_pred += preds.tolist()\n",
    "                    y_true += val.tolist()\n",
    "                y_true,y_pred=np.array(y_true), np.array(y_pred)\n",
    "                if IMPLICT:\n",
    "                    epoch_score = roc_auc_score(y_true,y_pred)\n",
    "                    score='auc'\n",
    "                else:\n",
    "                    epoch_score=sum([(y - x) ** 2 for x, y in zip(y_true, y_pred)]) / len(y_pred)\n",
    "                    score='mse'\n",
    "\n",
    "                # 计算top10的recall、precision、推荐物品覆盖率\n",
    "                user_item=loaders['valid'].dataset.user_item\n",
    "                items = torch.arange(self.n_items).long()\n",
    "                hit, rec_count, test_count,all_rec_items = 0,0,0,set()\n",
    "                train_ui=loaders['train'].dataset.user_item\n",
    "                for u in user_item:\n",
    "                    target_items=user_item[u]\n",
    "                    # seen_items = np.array(list(train_ui[u].keys()))\n",
    "\n",
    "                    users=[int(u)]*self.n_items\n",
    "                    users = torch.Tensor(users).long()\n",
    "                    scores=self.forward(users,items)\n",
    "                    if u in train_ui:\n",
    "                        seen_items = np.array(list(train_ui[u].keys()))\n",
    "                        scores[seen_items]=-1e9\n",
    "                    recs=np.argsort(scores)[-10:].tolist()\n",
    "\n",
    "                    for item in recs:  # 遍历给user推荐的物品\n",
    "                        if item in target_items:  # 测试集中有该物品\n",
    "                            hit += 1  # 推荐命中+1\n",
    "                        all_rec_items.add(item)\n",
    "                    rec_count += self.topn\n",
    "                    test_count += len(target_items)\n",
    "                    precision = hit / (1.0 * rec_count)\n",
    "                    recall = hit / (1.0 * test_count)\n",
    "                    coverage = len(all_rec_items) / (1.0 * self.n_items)\n",
    "\n",
    "\n",
    "            if ((epoch + 1) % 1) == 0:\n",
    "                print(\n",
    "                    f'epoch {epoch + 1} train loss: {losses[\"train\"]:.3f} valid loss {losses[\"valid\"]:.3f} {score} {epoch_score:.3f}')\n",
    "                print('precisioin=%.4f\\trecall=%.4f\\tcoverage=%.4f' % (precision, recall, coverage))\n",
    "\n",
    "            # if ((epoch + 1) % 1) == 0:\n",
    "            #     print(\n",
    "            #         f'epoch {epoch + 1} train loss: {losses[\"train\"]:.3f} valid loss {losses[\"valid\"]:.3f}')\n",
    "        return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T11:38:27.755929Z",
     "start_time": "2020-09-19T11:38:27.577380Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize end.The user number is:943,item number is:1682\n"
     ]
    }
   ],
   "source": [
    "# from lfm_new_data import getDataLoader,LFM\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from utils import Interactions\n",
    "import torch\n",
    "input_size, loader=getDataLoader(\"hin/UI.txt\", batch_size=2048)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T11:42:57.230973Z",
     "start_time": "2020-09-19T11:38:30.365753Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 1 ): 100%|████████████████████████████████████████████████████| 40/40 [00:35<00:00,  1.13it/s, train_loss=3.56]\n",
      "(valid: 1 ): 100%|█████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.14it/s, train_loss=3.1]\n",
      "(train: 2 ):   0%|                                                                              | 0/40 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 train loss: 15.878 valid loss 3.240 mse 3.240\n",
      "precisioin=0.0704\trecall=0.0331\tcoverage=0.4328\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 2 ): 100%|████████████████████████████████████████████████████| 40/40 [00:35<00:00,  1.13it/s, train_loss=1.41]\n",
      "(valid: 2 ): 100%|████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.13it/s, train_loss=1.31]\n",
      "(train: 3 ):   0%|                                                                              | 0/40 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2 train loss: 1.491 valid loss 1.296 mse 1.296\n",
      "precisioin=0.0989\trecall=0.0466\tcoverage=0.2771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 3 ): 100%|████████████████████████████████████████████████████| 40/40 [00:35<00:00,  1.12it/s, train_loss=1.07]\n",
      "(valid: 3 ): 100%|████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.13it/s, train_loss=1.13]\n",
      "(train: 4 ):   0%|                                                                              | 0/40 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3 train loss: 0.939 valid loss 1.103 mse 1.103\n",
      "precisioin=0.1115\trecall=0.0524\tcoverage=0.1683\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 4 ): 100%|███████████████████████████████████████████████████| 40/40 [00:35<00:00,  1.13it/s, train_loss=0.972]\n",
      "(valid: 4 ): 100%|████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.14it/s, train_loss=1.07]\n",
      "(train: 5 ):   0%|                                                                              | 0/40 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4 train loss: 0.891 valid loss 1.065 mse 1.065\n",
      "precisioin=0.1290\trecall=0.0607\tcoverage=0.1350\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 5 ): 100%|████████████████████████████████████████████████████| 40/40 [00:35<00:00,  1.14it/s, train_loss=1.15]\n",
      "(valid: 5 ): 100%|████████████████████████████████████████████████████| 10/10 [00:08<00:00,  1.13it/s, train_loss=1.15]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5 train loss: 0.885 valid loss 1.053 mse 1.053\n",
      "precisioin=0.1491\trecall=0.0702\tcoverage=0.1249\n"
     ]
    }
   ],
   "source": [
    "model = LFM(input_size[0],input_size[1])\n",
    "model.fit(loader,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T11:44:17.999390Z",
     "start_time": "2020-09-19T11:44:17.790943Z"
    }
   },
   "outputs": [],
   "source": [
    "pd.to_pickle(model,'UI.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T11:44:39.573847Z",
     "start_time": "2020-09-19T11:44:36.504057Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialize end.The user number is:943,item number is:1682\n"
     ]
    }
   ],
   "source": [
    "# from lfm_new_data import getDataLoader,LFM\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from utils import Interactions\n",
    "import torch\n",
    "input_size, loader=getDataLoader(\"hin/UIUI.txt\", batch_size=2048)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-09-19T12:56:56.058353Z",
     "start_time": "2020-09-19T11:44:47.133135Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 1 ): 100%|██████████████████████████████████████████████| 620/620 [09:32<00:00,  1.08it/s, train_loss=6.07e+10]\n",
      "(valid: 1 ): 100%|██████████████████████████████████████████████| 155/155 [02:22<00:00,  1.08it/s, train_loss=2.08e+10]\n",
      "(train: 2 ):   0%|                                                                             | 0/620 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 train loss: 40297200151.465 valid loss 34246581271.081 mse 34246581319.054\n",
      "precisioin=1.0000\trecall=0.0297\tcoverage=0.0583\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 2 ): 100%|██████████████████████████████████████████████| 620/620 [09:35<00:00,  1.08it/s, train_loss=1.46e+10]\n",
      "(valid: 2 ): 100%|███████████████████████████████████████████████| 155/155 [02:22<00:00,  1.09it/s, train_loss=9.99e+9]\n",
      "(train: 3 ):   0%|                                                                             | 0/620 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2 train loss: 26704211269.207 valid loss 20691389666.063 mse 20691389648.508\n",
      "precisioin=1.0000\trecall=0.0297\tcoverage=0.0606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 3 ): 100%|████████████████████████████████████████████████| 620/620 [09:30<00:00,  1.09it/s, train_loss=5.5e+9]\n",
      "(valid: 3 ): 100%|███████████████████████████████████████████████| 155/155 [02:21<00:00,  1.10it/s, train_loss=4.77e+9]\n",
      "(train: 4 ):   0%|                                                                             | 0/620 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3 train loss: 15684147468.663 valid loss 12014367207.357 mse 12014367187.675\n",
      "precisioin=1.0000\trecall=0.0297\tcoverage=0.0606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 4 ): 100%|███████████████████████████████████████████████| 620/620 [09:39<00:00,  1.07it/s, train_loss=7.65e+9]\n",
      "(valid: 4 ): 100%|███████████████████████████████████████████████| 155/155 [02:22<00:00,  1.09it/s, train_loss=2.53e+9]\n",
      "(train: 5 ):   0%|                                                                             | 0/620 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4 train loss: 8986983976.212 valid loss 6915117062.965 mse 6915117083.236\n",
      "precisioin=1.0000\trecall=0.0297\tcoverage=0.0618\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "(train: 5 ): 100%|███████████████████████████████████████████████| 620/620 [09:56<00:00,  1.04it/s, train_loss=3.99e+9]\n",
      "(valid: 5 ): 100%|███████████████████████████████████████████████| 155/155 [02:27<00:00,  1.05it/s, train_loss=1.51e+9]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5 train loss: 5120800658.736 valid loss 3981462963.339 mse 3981462974.818\n",
      "precisioin=1.0000\trecall=0.0297\tcoverage=0.0612\n"
     ]
    }
   ],
   "source": [
    "model = LFM(input_size[0],input_size[1],n_factors=5)\n",
    "model.fit(loader,5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.to_pickle(model,'UIUI.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
